In [191]:
import pandas as pd
import numpy as np

from sklearn.ensemble import VotingClassifier

from sklearn.metrics import roc_curve, auc, precision_recall_curve, average_precision_score
from sklearn.preprocessing import label_binarize
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neural_network import MLPClassifier
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier

from sklearn.metrics import (
    classification_report, roc_auc_score, roc_curve, precision_recall_curve, f1_score,
    matthews_corrcoef, confusion_matrix, accuracy_score, precision_score, recall_score, auc
)
from sklearn.base import clone
from sklearn.model_selection import cross_val_score
from sklearn.metrics import auc as calculate_auc
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.model_selection import GridSearchCV, StratifiedKFold, learning_curve
from sklearn.experimental import enable_halving_search_cv  # 👈 REQUIRED to use HalvingGridSearchCV
from sklearn.model_selection import HalvingGridSearchCV
import shap
# Ignore warnings for clean output
import warnings
warnings.filterwarnings('ignore')
from sklearn.preprocessing import label_binarize
from sklearn.base import clone
In [2]:
X_train_multi = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\balanced_X_train_multi.csv")
y_train_multi = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\balanced_y_train_multi.csv").squeeze()
X_test = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\X_test.csv")
y_test_multi = pd.read_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\processed\y_test_multi.csv").squeeze()


# Verify loaded data
print("Datasets Loaded:")
print(f"X_train_multi: {X_train_multi.shape}")
print(f"y_train_multi: {y_train_multi.shape}")
print(f"X_test: {X_test.shape}")
print(f"y_test_multi: {y_test_multi.shape}")
Datasets Loaded:
X_train_multi: (330556, 7)
y_train_multi: (330556,)
X_test: (52487, 7)
y_test_multi: (52487,)
In [3]:
# For y_train_multi
if isinstance(y_train_multi, pd.DataFrame) and y_train_multi.shape[1] == 1:
    y_train_multi = y_train_multi.values.ravel()
elif isinstance(y_train_multi, pd.Series):
    y_train_multi = y_train_multi.values
elif isinstance(y_train_multi, np.ndarray) and y_train_multi.ndim > 1:
    y_train_multi = y_train_multi.ravel()

# For y_test_multi
if isinstance(y_test_multi, pd.DataFrame) and y_test_multi.shape[1] == 1:
    y_test_multi = y_test_multi.values.ravel()
elif isinstance(y_test_multi, pd.Series):
    y_test_multi = y_test_multi.values
elif isinstance(y_test_multi, np.ndarray) and y_test_multi.ndim > 1:
    y_test_multi = y_test_multi.ravel()

# === Verification ===
print("Datasets Loaded and Verified:")
print(f"X_train_multi: {X_train_multi.shape}")
print(f"y_train_multi: {y_train_multi.shape}")
print(f"X_test: {X_test.shape}")
print(f"y_test_multi: {y_test_multi.shape}")
Datasets Loaded and Verified:
X_train_multi: (330556, 7)
y_train_multi: (330556,)
X_test: (52487, 7)
y_test_multi: (52487,)
In [8]:
models = {
    "Decision Tree": DecisionTreeClassifier(random_state=42),
    "Random Forest": RandomForestClassifier(random_state=42),
    "XGBoost": XGBClassifier(objective="multi:softprob", num_class=5, use_label_encoder=False, eval_metric="mlogloss", verbosity=0),
    "LightGBM": LGBMClassifier(objective="multiclass", num_class=5),
    "Gradient Boosting": GradientBoostingClassifier(random_state=42),
    "Logistic Regression": LogisticRegression(multi_class="multinomial", solver="lbfgs", max_iter=1000),
    "MLP": MLPClassifier(max_iter=1000, random_state=42)
}
In [10]:
param_grid_xgb = {"n_estimators": [50], "max_depth": [3, 5], "learning_rate": [0.05, 0.1]}
param_grid_lgb = {"n_estimators": [50], "max_depth": [5, 10], "learning_rate": [0.05, 0.1]}
param_grid_dt = {"max_depth": [None, 5, 10], "min_samples_split": [2, 5]}
param_grid_rf = {"n_estimators": [50, 100], "max_depth": [None, 5, 10], "min_samples_split": [2, 5]}
param_grid_gb = {"n_estimators": [50, 100], "learning_rate": [0.05, 0.1], "max_depth": [3, 5]}
param_grid_lr = {"C": [0.1, 1.0, 10.0], "penalty": ["l2"]}
param_grid_mlp = {"hidden_layer_sizes": [(100,), (50, 50)], "activation": ["relu", "tanh"], "learning_rate_init": [0.001]}
In [12]:
models_with_grids = {
    "Decision Tree": {"model": models["Decision Tree"], "param_grid": param_grid_dt},
    "Random Forest": {"model": models["Random Forest"], "param_grid": param_grid_rf},
    "XGBoost": {"model": models["XGBoost"], "param_grid": param_grid_xgb},
    "LightGBM": {"model": models["LightGBM"], "param_grid": param_grid_lgb},
    "Gradient Boosting": {"model": models["Gradient Boosting"], "param_grid": param_grid_gb},
    "Logistic Regression": {"model": models["Logistic Regression"], "param_grid": param_grid_lr},
     "MLP": {"model": models["MLP"], "param_grid": param_grid_mlp}
}
In [14]:
cv_strategy = StratifiedKFold(n_splits=3, shuffle=True, random_state=42)
In [16]:
def run_multiclass_grid_search(X_train, y_train, X_test, y_test, models_with_grids, cv_strategy, scoring="f1_macro"):
    import pandas as pd
    import numpy as np
    from sklearn.metrics import (
        classification_report, accuracy_score, f1_score, matthews_corrcoef,
        confusion_matrix, precision_recall_curve, roc_auc_score, auc
    )
    from sklearn.model_selection import GridSearchCV
    from itertools import product

    grid_search_results = {}
    trained_models = {}
    threshold = 0.1  # 10% gap for over/under-fitting check

    def check_over_under(train_acc, test_acc):
        if train_acc > test_acc + threshold:
            return "Potential Overfitting Detected"
        elif test_acc > train_acc + threshold:
            return "Potential Underfitting Detected"
        else:
            return "Balanced Performance"

    def manual_search(model_class, param_grid, model_name):
        print(f"\nRunning manual grid search for {model_name}...")
        keys, values = zip(*param_grid.items())
        best_score = -np.inf
        best_model = None
        best_params = None

        for combo in product(*values):
            params = dict(zip(keys, combo))
            try:
                model = model_class(**params)
                model.fit(X_train, y_train)
                preds = model.predict(X_test)
                score = f1_score(y_test, preds, average="macro")
                
                if score > best_score:
                    best_score = score
                    best_model = model
                    best_params = params
            except Exception as e:
                print(f"Skipping {params} due to error: {e}")

        return best_model, best_params

    for model_name, model_data in models_with_grids.items():
        print(f"\nStarting grid search for {model_name}...")
        model = model_data["model"]
        param_grid = model_data["param_grid"]

        if model_name in ["XGBoost", "LightGBM"]:
            best_model, best_params = manual_search(model.__class__, param_grid, model_name)
        else:
            grid_search = GridSearchCV(
                estimator=model,
                param_grid=param_grid,
                scoring=scoring,
                cv=cv_strategy,
                verbose=2,
                n_jobs=-1
            )
            grid_search.fit(X_train, y_train)
            best_model = grid_search.best_estimator_
            best_params = grid_search.best_params_

        trained_models[model_name] = best_model
        print(f"Best Parameters for {model_name}: {best_params}")

        y_train_pred = best_model.predict(X_train)
        y_test_pred = best_model.predict(X_test)

        train_acc = accuracy_score(y_train, y_train_pred)
        test_acc = accuracy_score(y_test, y_test_pred)
        fit_status = check_over_under(train_acc, test_acc)

        print(f"Train Accuracy: {train_acc:.4f}")
        print(f"Test Accuracy: {test_acc:.4f}")
        print(f"{fit_status}")

        f1 = f1_score(y_test, y_test_pred, average="macro")
        mcc = matthews_corrcoef(y_test, y_test_pred)
        cm = confusion_matrix(y_test, y_test_pred)
        specificity = cm[0, 0] / np.sum(cm[0]) if cm.shape[0] > 1 else None

        auc_roc = auc_pr = None
        if hasattr(best_model, "predict_proba"):
            try:
                y_probs = best_model.predict_proba(X_test)
                auc_roc = roc_auc_score(y_test, y_probs, multi_class='ovr')
                auc_pr = np.mean([
                    auc(*precision_recall_curve((y_test == i).astype(int), y_probs[:, i])[1::-1])
                    for i in np.unique(y_test)
                ])
            except Exception as e:
                print(f"AUC calculation failed for {model_name}: {e}")

        print(f"\nClassification Report for {model_name}:\n", classification_report(y_test, y_test_pred))

        grid_search_results[model_name] = {
            "Best Parameters": best_params,
            "Train Accuracy": train_acc,
            "Test Accuracy": test_acc,
            "F1-Score": f1,
            "AUC-PR": auc_pr,
            "AUC-ROC": auc_roc,
            "MCC": mcc,
            "Specificity": specificity,
            "Fit Status": fit_status
        }

        print(f"Results for {model_name}:")
        print(grid_search_results[model_name])

    results_df = pd.DataFrame(grid_search_results).T
    print("\nSummary of All Model Results:")
    print(results_df)

    return trained_models, grid_search_results, results_df
In [18]:
trained_models, grid_results, summary_df = run_multiclass_grid_search(
    X_train=X_train_multi,
    y_train=y_train_multi,
    X_test=X_test,
    y_test=y_test_multi,
    models_with_grids=models_with_grids,
    cv_strategy=cv_strategy,
    scoring="f1_macro"
)
Starting grid search for Decision Tree...
Fitting 3 folds for each of 6 candidates, totalling 18 fits
Best Parameters for Decision Tree: {'max_depth': None, 'min_samples_split': 2}
Train Accuracy: 1.0000
Test Accuracy: 0.7544
Potential Overfitting Detected

Classification Report for Decision Tree:
               precision    recall  f1-score   support

           0       0.29      0.44      0.35      2683
           1       0.95      0.82      0.88     38015
           2       0.05      0.17      0.08       797
           3       0.77      0.85      0.81      5512
           4       0.35      0.44      0.39      5480

    accuracy                           0.75     52487
   macro avg       0.48      0.54      0.50     52487
weighted avg       0.82      0.75      0.78     52487

Results for Decision Tree:
{'Best Parameters': {'max_depth': None, 'min_samples_split': 2}, 'Train Accuracy': 1.0, 'Test Accuracy': 0.7543772743727019, 'F1-Score': 0.5014763838607188, 'AUC-PR': 0.5377037520327916, 'AUC-ROC': 0.737595282931572, 'MCC': 0.5350155226346356, 'Specificity': 0.4383153186731271, 'Fit Status': 'Potential Overfitting Detected'}

Starting grid search for Random Forest...
Fitting 3 folds for each of 12 candidates, totalling 36 fits
Best Parameters for Random Forest: {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 100}
Train Accuracy: 1.0000
Test Accuracy: 0.8104
Potential Overfitting Detected

Classification Report for Random Forest:
               precision    recall  f1-score   support

           0       0.46      0.51      0.48      2683
           1       0.95      0.87      0.91     38015
           2       0.06      0.10      0.08       797
           3       0.82      0.90      0.86      5512
           4       0.41      0.57      0.48      5480

    accuracy                           0.81     52487
   macro avg       0.54      0.59      0.56     52487
weighted avg       0.84      0.81      0.82     52487

Results for Random Forest:
{'Best Parameters': {'max_depth': None, 'min_samples_split': 2, 'n_estimators': 100}, 'Train Accuracy': 1.0, 'Test Accuracy': 0.8104101968106389, 'F1-Score': 0.5605885000054511, 'AUC-PR': 0.5823549331033362, 'AUC-ROC': 0.9107117889577276, 'MCC': 0.6203093295989356, 'Specificity': 0.506895266492732, 'Fit Status': 'Potential Overfitting Detected'}

Starting grid search for XGBoost...

Running manual grid search for XGBoost...
Best Parameters for XGBoost: {'n_estimators': 50, 'max_depth': 5, 'learning_rate': 0.1}
Train Accuracy: 0.7086
Test Accuracy: 0.7408
Balanced Performance

Classification Report for XGBoost:
               precision    recall  f1-score   support

           0       0.51      0.54      0.53      2683
           1       0.97      0.80      0.87     38015
           2       0.05      0.56      0.09       797
           3       0.80      0.90      0.85      5512
           4       0.52      0.30      0.38      5480

    accuracy                           0.74     52487
   macro avg       0.57      0.62      0.54     52487
weighted avg       0.86      0.74      0.79     52487

Results for XGBoost:
{'Best Parameters': {'n_estimators': 50, 'max_depth': 5, 'learning_rate': 0.1}, 'Train Accuracy': 0.7086030808698073, 'Test Accuracy': 0.7407929582563302, 'F1-Score': 0.544558698061361, 'AUC-PR': 0.5950468273043812, 'AUC-ROC': 0.9176916304982848, 'MCC': 0.5475955078822186, 'Specificity': 0.5415579575102497, 'Fit Status': 'Balanced Performance'}

Starting grid search for LightGBM...

Running manual grid search for LightGBM...
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.011710 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009163 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010699 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009463 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1785
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 7
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
Best Parameters for LightGBM: {'n_estimators': 50, 'max_depth': 10, 'learning_rate': 0.1}
Train Accuracy: 0.7510
Test Accuracy: 0.7544
Balanced Performance

Classification Report for LightGBM:
               precision    recall  f1-score   support

           0       0.49      0.55      0.52      2683
           1       0.97      0.81      0.88     38015
           2       0.05      0.44      0.09       797
           3       0.81      0.91      0.86      5512
           4       0.47      0.37      0.42      5480

    accuracy                           0.75     52487
   macro avg       0.55      0.62      0.55     52487
weighted avg       0.86      0.75      0.80     52487

Results for LightGBM:
{'Best Parameters': {'n_estimators': 50, 'max_depth': 10, 'learning_rate': 0.1}, 'Train Accuracy': 0.7509862171613887, 'Test Accuracy': 0.75441537904624, 'F1-Score': 0.5515070952932931, 'AUC-PR': 0.5928606148635792, 'AUC-ROC': 0.9178655782468033, 'MCC': 0.561307202499785, 'Specificity': 0.554230339172568, 'Fit Status': 'Balanced Performance'}

Starting grid search for Gradient Boosting...
Fitting 3 folds for each of 8 candidates, totalling 24 fits
Best Parameters for Gradient Boosting: {'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 100}
Train Accuracy: 0.7718
Test Accuracy: 0.7542
Balanced Performance

Classification Report for Gradient Boosting:
               precision    recall  f1-score   support

           0       0.47      0.55      0.51      2683
           1       0.97      0.80      0.88     38015
           2       0.05      0.38      0.09       797
           3       0.80      0.91      0.85      5512
           4       0.43      0.42      0.43      5480

    accuracy                           0.75     52487
   macro avg       0.54      0.61      0.55     52487
weighted avg       0.85      0.75      0.80     52487

Results for Gradient Boosting:
{'Best Parameters': {'learning_rate': 0.1, 'max_depth': 5, 'n_estimators': 100}, 'Train Accuracy': 0.771838962233328, 'Test Accuracy': 0.7541867510050108, 'F1-Score': 0.5495046428954449, 'AUC-PR': 0.5896444483046593, 'AUC-ROC': 0.9150207845151066, 'MCC': 0.5603425328386388, 'Specificity': 0.5501304509877003, 'Fit Status': 'Balanced Performance'}

Starting grid search for Logistic Regression...
Fitting 3 folds for each of 3 candidates, totalling 9 fits
Best Parameters for Logistic Regression: {'C': 1.0, 'penalty': 'l2'}
Train Accuracy: 0.6714
Test Accuracy: 0.7140
Balanced Performance

Classification Report for Logistic Regression:
               precision    recall  f1-score   support

           0       0.44      0.56      0.50      2683
           1       0.96      0.77      0.85     38015
           2       0.05      0.53      0.09       797
           3       0.76      0.88      0.82      5512
           4       0.43      0.28      0.34      5480

    accuracy                           0.71     52487
   macro avg       0.53      0.60      0.52     52487
weighted avg       0.85      0.71      0.77     52487

Results for Logistic Regression:
{'Best Parameters': {'C': 1.0, 'penalty': 'l2'}, 'Train Accuracy': 0.6713658200123428, 'Test Accuracy': 0.7140053727589689, 'F1-Score': 0.5187632053437157, 'AUC-PR': 0.5698716080382211, 'AUC-ROC': 0.9038811531490225, 'MCC': 0.5145789572844949, 'Specificity': 0.5624301155423034, 'Fit Status': 'Balanced Performance'}

Starting grid search for MLP...
Fitting 3 folds for each of 4 candidates, totalling 12 fits
Best Parameters for MLP: {'activation': 'tanh', 'hidden_layer_sizes': (50, 50), 'learning_rate_init': 0.001}
Train Accuracy: 0.8277
Test Accuracy: 0.7533
Balanced Performance

Classification Report for MLP:
               precision    recall  f1-score   support

           0       0.40      0.52      0.45      2683
           1       0.96      0.80      0.87     38015
           2       0.04      0.19      0.07       797
           3       0.79      0.88      0.83      5512
           4       0.35      0.50      0.41      5480

    accuracy                           0.75     52487
   macro avg       0.51      0.58      0.53     52487
weighted avg       0.84      0.75      0.79     52487

Results for MLP:
{'Best Parameters': {'activation': 'tanh', 'hidden_layer_sizes': (50, 50), 'learning_rate_init': 0.001}, 'Train Accuracy': 0.8277266181826982, 'Test Accuracy': 0.7533103435136319, 'F1-Score': 0.528425428935922, 'AUC-PR': 0.5500542790039462, 'AUC-ROC': 0.8882511238338943, 'MCC': 0.5518174569625105, 'Specificity': 0.5218039508013418, 'Fit Status': 'Balanced Performance'}

Summary of All Model Results:
                                                       Best Parameters  \
Decision Tree              {'max_depth': None, 'min_samples_split': 2}   
Random Forest        {'max_depth': None, 'min_samples_split': 2, 'n...   
XGBoost              {'n_estimators': 50, 'max_depth': 5, 'learning...   
LightGBM             {'n_estimators': 50, 'max_depth': 10, 'learnin...   
Gradient Boosting    {'learning_rate': 0.1, 'max_depth': 5, 'n_esti...   
Logistic Regression                        {'C': 1.0, 'penalty': 'l2'}   
MLP                  {'activation': 'tanh', 'hidden_layer_sizes': (...   

                    Train Accuracy Test Accuracy  F1-Score    AUC-PR  \
Decision Tree                  1.0      0.754377  0.501476  0.537704   
Random Forest                  1.0       0.81041  0.560589  0.582355   
XGBoost                   0.708603      0.740793  0.544559  0.595047   
LightGBM                  0.750986      0.754415  0.551507  0.592861   
Gradient Boosting         0.771839      0.754187  0.549505  0.589644   
Logistic Regression       0.671366      0.714005  0.518763  0.569872   
MLP                       0.827727       0.75331  0.528425  0.550054   

                      AUC-ROC       MCC Specificity  \
Decision Tree        0.737595  0.535016    0.438315   
Random Forest        0.910712  0.620309    0.506895   
XGBoost              0.917692  0.547596    0.541558   
LightGBM             0.917866  0.561307     0.55423   
Gradient Boosting    0.915021  0.560343     0.55013   
Logistic Regression  0.903881  0.514579     0.56243   
MLP                  0.888251  0.551817    0.521804   

                                         Fit Status  
Decision Tree        Potential Overfitting Detected  
Random Forest        Potential Overfitting Detected  
XGBoost                        Balanced Performance  
LightGBM                       Balanced Performance  
Gradient Boosting              Balanced Performance  
Logistic Regression            Balanced Performance  
MLP                            Balanced Performance  
In [128]:
import os
import joblib

def save_models_individually(model_dict, save_dir):
    """
    Saves each model from the given dictionary to a .pkl file in the specified directory.
    Filenames are derived from model names.
    """
    os.makedirs(save_dir, exist_ok=True)
    
    for model_name, model_obj in model_dict.items():
        # Clean filename
        clean_name = model_name.replace(" ", "_").replace("(", "").replace(")", "").replace("-", "")
        filename = os.path.join(save_dir, f"{clean_name}.pkl")
        joblib.dump(model_obj, filename)
        print(f"Saved: {filename}")

save_models_individually(
    model_dict=trained_models,
    save_dir=r"C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch"
)
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Decision_Tree.pkl
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Random_Forest.pkl
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\XGBoost.pkl
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\LightGBM.pkl
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Gradient_Boosting.pkl
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\Logistic_Regression.pkl
Saved: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Multi Training - GridSearch\MLP.pkl
In [61]:
def plot_model_metric_bars(results_df):
    """
    Plots grouped bar charts for each metric:
    Train vs Test values side-by-side per model.
    """
    # Define metrics (both train/test where applicable)
    metric_groups = [
        ("Train Accuracy", "Test Accuracy", "Accuracy"),
        ("F1-Score", "F1-Score", "F1-Macro"),
        ("AUC-PR", "AUC-PR", "AUC-PR"),
        ("AUC-ROC", "AUC-ROC", "AUC-ROC")
    ]

    color_train = "lightsteelblue"
    color_test = "peachpuff"

    for train_col, test_col, display_name in metric_groups:
        if train_col not in results_df.columns or test_col not in results_df.columns:
            print(f"Skipping {display_name}: missing columns.")
            continue

        x = np.arange(len(results_df))
        width = 0.35

        fig, ax = plt.subplots(figsize=(10, 6))
        bars_train = ax.bar(x - width / 2, results_df[train_col], width, label="Train", color=color_train)
        bars_test = ax.bar(x + width / 2, results_df[test_col], width, label="Test", color=color_test)

        # Add data labels
        for bars in [bars_train, bars_test]:
            for bar in bars:
                height = bar.get_height()
                ax.text(
                    bar.get_x() + bar.get_width() / 2.0,
                    height,
                    f"{height:.2f}",
                    ha="center", va="bottom", fontsize=9
                )

        ax.set_ylabel(display_name)
        ax.set_title(f"{display_name} Comparison (Train vs Test)")
        ax.set_xticks(x)
        ax.set_xticklabels(results_df.index, rotation=45, ha="right")
        ax.legend()
        plt.tight_layout()
        plt.show()
In [62]:
plot_model_metric_bars(
    results_df=summary_df  # From your run_multiclass_grid_search
)
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [111]:
def plot_roc_pr_curves(model_dict, X_test, y_test, title_prefix="Model Evaluation"):
    class_labels = np.unique(y_test)
    y_test_bin = label_binarize(y_test, classes=class_labels)

    for name, model in model_dict.items():
        if hasattr(model, "predict_proba"):
            try:
                y_proba = model.predict_proba(X_test)

                # === ROC Curve ===
                plt.figure(figsize=(8, 5))
                for i in range(len(class_labels)):
                    fpr, tpr, _ = roc_curve(y_test_bin[:, i], y_proba[:, i])
                    roc_auc = auc(fpr, tpr)
                    plt.plot(fpr, tpr, label=f"Class {class_labels[i]} (AUC = {roc_auc:.2f})")
                plt.plot([0, 1], [0, 1], 'k--')
                plt.title(f"{title_prefix} - ROC Curve: {name}")
                plt.xlabel("False Positive Rate")
                plt.ylabel("True Positive Rate")
                plt.legend(loc="lower right", fontsize='small')
                plt.grid(True)
                plt.tight_layout()
                plt.show()

                # === Precision-Recall Curve ===
                plt.figure(figsize=(8, 5))
                for i in range(len(class_labels)):
                    precision, recall, _ = precision_recall_curve(y_test_bin[:, i], y_proba[:, i])
                    ap = average_precision_score(y_test_bin[:, i], y_proba[:, i])
                    plt.plot(recall, precision, label=f"Class {class_labels[i]} (AP = {ap:.2f})")
                plt.title(f"{title_prefix} - Precision-Recall Curve: {name}")
                plt.xlabel("Recall")
                plt.ylabel("Precision")
                plt.legend(loc="lower left", fontsize='small')
                plt.grid(True)
                plt.tight_layout()
                plt.show()

            except Exception as e:
                print(f"Error plotting for {name}: {e}")
In [115]:
plot_roc_pr_curves(
    model_dict=trained_models,  # or all_models
    X_test=X_test,
    y_test=y_test_multi,
    title_prefix="Initial Grid Search"
)
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [52]:
def explain_models_with_shap(models, X_train, X_test, sample_size=100, background_size=100):
    import shap
    import numpy as np
    import pandas as pd

    shap_values_main = {}
    X_test_sample = X_test.sample(sample_size, random_state=42)
    background = shap.sample(X_train, background_size, random_state=42)

    for model_name, model in models.items():
        print(f"\nSHAP for: {model_name}")
        try:
            # === Tree-based models with TreeExplainer ===
            if model_name.lower().startswith(("xgboost", "lightgbm", "random forest")):
                # Use model-specific booster when possible
                explainer = shap.TreeExplainer(model)
                shap_vals = explainer.shap_values(X_test_sample)

            # === All others use KernelExplainer ===
            else:
                explainer = shap.KernelExplainer(model.predict_proba, background)
                shap_vals = explainer.shap_values(X_test_sample)

            # Handle multiclass
            if isinstance(shap_vals, list):
                shap_vals = shap_vals[1] if len(shap_vals) > 1 else shap_vals[0]

            shap_vals_main = shap_vals[:, :, 1] if shap_vals.ndim == 3 else shap_vals

            if shap_vals_main.shape[0] != X_test_sample.shape[0]:
                print(f"Mismatch in SHAP shape for {model_name}")
                continue

            shap_values_main[model_name] = shap_vals_main
            shap.summary_plot(shap_vals_main, X_test_sample, feature_names=X_train.columns, plot_type="dot", show=True)

        except Exception as e:
            print(f"SHAP failed for {model_name}: {e}")
    return shap_values_main

def plot_mean_shap_summary(shap_values_main, feature_names):

    shap_summary_df = pd.DataFrame({
        model: np.abs(values).mean(axis=0)
        for model, values in shap_values_main.items()
    }, index=feature_names).T

    mean_shap = shap_summary_df.mean(axis=0).sort_values(ascending=False)

    plt.figure(figsize=(10, 6))
    ax = sns.barplot(x=mean_shap.values, y=mean_shap.index, palette="Set3")
    for rect in ax.patches:
        width = rect.get_width()
        ax.text(width + 0.001, rect.get_y() + rect.get_height() / 2, f"{width:.3f}", ha="left", va="center")
    plt.title("Mean Absolute SHAP Value (Across All Models)")
    plt.xlabel("SHAP Value")
    plt.tight_layout()
    plt.show()
In [54]:
# Explain and plot individual SHAP summary plots
shap_values_main = explain_models_with_shap(
    models=trained_models,
    X_train=X_train_multi,
    X_test=X_test,
    sample_size=100,
    background_size=100
)

# Plot overall SHAP feature importance
plot_mean_shap_summary(
    shap_values_main=shap_values_main,
    feature_names=X_train_multi.columns
)
SHAP for: Decision Tree
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Random Forest
No description has been provided for this image
SHAP for: XGBoost
No description has been provided for this image
SHAP for: LightGBM
No description has been provided for this image
SHAP for: Gradient Boosting
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Logistic Regression
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: MLP
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
No description has been provided for this image
In [56]:
from sklearn.model_selection import learning_curve
import pandas as pd

def get_learning_curve_data(estimator, X, y, cv, scoring="f1_macro", n_jobs=-1, train_sizes=np.linspace(0.1, 1.0, 5)):
    """
    Returns a DataFrame with training size, train and CV scores (mean and std) for a given model.
    """
    train_sizes, train_scores, cv_scores = learning_curve(
        estimator,
        X,
        y,
        cv=cv,
        scoring=scoring,
        n_jobs=n_jobs,
        train_sizes=train_sizes,
        shuffle=True,
        random_state=42
    )

    df = pd.DataFrame({
        "Training Size": train_sizes,
        "Train Score Mean": train_scores.mean(axis=1),
        "Train Score Std": train_scores.std(axis=1),
        "CV Score Mean": cv_scores.mean(axis=1),
        "CV Score Std": cv_scores.std(axis=1)
    })

    return df
In [58]:
# Create the dictionary
learning_curve_data_dict = {}

# You can use your cv_strategy from before (e.g., StratifiedKFold)
for model_name, model in trained_models.items():
    print(f"Generating learning curve for: {model_name}")
    try:
        df_lc = get_learning_curve_data(
            estimator=model,
            X=X_train_multi,
            y=y_train_multi,
            cv=cv_strategy,
            scoring="f1_macro"
        )
        learning_curve_data_dict[model_name] = df_lc
    except Exception as e:
        print(f"Failed for {model_name}: {e}")
Generating learning curve for: Decision Tree
Generating learning curve for: Random Forest
Generating learning curve for: XGBoost
Failed for XGBoost: 'super' object has no attribute '__sklearn_tags__'
Generating learning curve for: LightGBM
Generating learning curve for: Gradient Boosting
Generating learning curve for: Logistic Regression
Generating learning curve for: MLP
In [59]:
def plot_combined_learning_curves(learning_curve_data_dict, title_prefix="Learning Curve"):
    for model_name, df in learning_curve_data_dict.items():
        if not isinstance(df, pd.DataFrame):
            continue
        x = df["Training Size"].values
        plt.figure(figsize=(10, 5))
        plt.plot(x, df["Train Score Mean"], 'o-', label="Train", color='tomato')
        plt.fill_between(x, df["Train Score Mean"] - df["Train Score Std"], df["Train Score Mean"] + df["Train Score Std"], alpha=0.1, color='tomato')
        plt.plot(x, df["CV Score Mean"], 's--', label="CV", color='seagreen')
        plt.fill_between(x, df["CV Score Mean"] - df["CV Score Std"], df["CV Score Mean"] + df["CV Score Std"], alpha=0.1, color='seagreen')
        plt.title(f"{title_prefix} - {model_name}")
        plt.xlabel("Training Examples")
        plt.ylabel("F1 Score")
        plt.legend()
        plt.grid(True)
        plt.tight_layout()
        plt.show()
In [60]:
plot_combined_learning_curves(
    learning_curve_data_dict=learning_curve_data_dict,
    title_prefix="Initial Grid Search"
)
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [67]:
regularised_models = {
    "Decision Tree": DecisionTreeClassifier(max_depth=5, min_samples_split=10, ccp_alpha=0.01, random_state=42),
    "Random Forest": RandomForestClassifier(n_estimators=100, max_depth=8, min_samples_leaf=5, min_samples_split=10, random_state=42),
    "XGBoost": XGBClassifier(n_estimators=50, max_depth=4, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, objective="multi:softprob", num_class=5, use_label_encoder=False, eval_metric="mlogloss", verbosity=0, random_state=42),
    "LightGBM": LGBMClassifier(n_estimators=50, max_depth=6, learning_rate=0.05, subsample=0.8, colsample_bytree=0.8, objective="multiclass", num_class=5, random_state=42),
    "Gradient Boosting": GradientBoostingClassifier(n_estimators=100, learning_rate=0.05, max_depth=4, subsample=0.8, random_state=42),
    "Logistic Regression": LogisticRegression(C=0.05, penalty="l2", multi_class="multinomial", solver="lbfgs", max_iter=1000, random_state=42),
    "MLP": MLPClassifier(hidden_layer_sizes=(50, 50), activation="tanh", alpha=0.01, learning_rate_init=0.001, max_iter=300, early_stopping=True, random_state=42)
}

fine_tune_grids = {
    "Decision Tree": {
        "max_depth": [None, 10, 20],
        "min_samples_split": [2, 5, 10],
    },
    "Random Forest": {
        "n_estimators": [100, 200],
        "max_depth": [None, 10],
        "min_samples_split": [2, 5],
    },
    "XGBoost": {
        "n_estimators": [50, 100],
        "max_depth": [3, 5, 7],
        "learning_rate": [0.05, 0.1],
    },
    "LightGBM": {
        "n_estimators": [50, 100],
        "max_depth": [5, 10],
        "learning_rate": [0.05, 0.1],
    },
    "Gradient Boosting": {
        "n_estimators": [100, 200],
        "max_depth": [3, 5],
        "learning_rate": [0.05, 0.1],
    },
    "Logistic Regression": {
        "C": [0.1, 1.0, 10],
        "penalty": ["l2"]
    },
    "MLP": {
        "hidden_layer_sizes": [(50,), (50, 50)],
        "activation": ["relu", "tanh"],
        "learning_rate_init": [0.001, 0.01]
    }
}
In [87]:
# Drop Coolant_Temperature from training and test sets
X_train_multi_dropped = X_train_multi.drop(columns=["Coolant_Temperature"])
X_test_dropped = X_test.drop(columns=["Coolant_Temperature"])
In [93]:
# Save the dropped feature datasets to CSV
X_train_multi_dropped.to_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\dropped_final_training\X_train_multi_dropped.csv", index=False)
X_test_dropped.to_csv(r"C:\Users\ghaza\Desktop\FYP\Application\AI\data\dropped_final_training\X_test_multi_dropped.csv", index=False)

print("Files saved successfully.")
Files saved successfully.
In [150]:
from itertools import product
import pandas as pd
import numpy as np
from sklearn.base import clone, is_classifier
from sklearn.model_selection import StratifiedKFold, cross_val_score

def safe_cross_val_score(estimator, X, y, cv, scoring):
    try:
        if not is_classifier(estimator):
            return np.array([np.nan])
        return cross_val_score(estimator, X, y, cv=cv, scoring=scoring, n_jobs=-1)
    except Exception as e:
        print(f"Cross-validation failed: {e}")
        return np.array([np.nan])
In [162]:
def evaluate_pipeline_multiclass_with_manual_tuning(
    regularised_models, fine_tune_grids, X_train, y_train, X_test, y_test, cv_folds=5
):
    cv = StratifiedKFold(n_splits=cv_folds, shuffle=True, random_state=42)

    all_models = {}
    reg_results, ft_results = {}, {}
    cm_dict_reg, cm_dict_ft = {}, {}

    def print_summary(name, model, X_train, y_train, X_test, y_test, best_params, fit_status, auc_pr, auc_roc, mcc, specificity):
        y_pred = model.predict(X_test)
        print(f"\nStarting grid search for {name}...")
        print(classification_report(y_test, y_pred))
        print(f"\nResults for {name}:")
        results = {
            "Best Parameters": best_params,
            "Train Accuracy": accuracy_score(y_train, model.predict(X_train)),
            "Test Accuracy": accuracy_score(y_test, y_pred),
            "F1-Score": f1_score(y_test, y_pred, average="macro"),
            "AUC-PR": auc_pr,
            "AUC-ROC": auc_roc,
            "MCC": mcc,
            "Specificity": specificity,
            "Fit Status": fit_status
        }
        for k, v in results.items():
            print(f"{k}: {v}")
        print("-" * 80)

    for model_name, base_model in regularised_models.items():
        print(f"\nProcessing {model_name}...")

        # === REGULARISED ===
        reg_model = clone(base_model)
        reg_model.fit(X_train, y_train)

        cv_scores = safe_cross_val_score(reg_model, X_train, y_train, cv=cv, scoring="f1_macro")
        y_train_pred = reg_model.predict(X_train)
        y_test_pred = reg_model.predict(X_test)

        train_acc = accuracy_score(y_train, y_train_pred)
        test_acc = accuracy_score(y_test, y_test_pred)

        fit_status = (
            "Potential Overfitting Detected!" if train_acc > test_acc + 0.1 else
            "Potential Underfitting Detected!" if test_acc > train_acc + 0.1 else
            "Balanced Performance"
        )

        auc_roc = auc_pr = None
        if hasattr(reg_model, "predict_proba"):
            try:
                y_probs = reg_model.predict_proba(X_test)
                auc_roc = roc_auc_score(y_test, y_probs, multi_class="ovr")
                auc_pr = np.mean([
                    auc(*precision_recall_curve((y_test == i).astype(int), y_probs[:, i])[1::-1])
                    for i in np.unique(y_test)
                ])
            except Exception:
                pass

        cm = confusion_matrix(y_test, y_test_pred)
        specificity = cm[0, 0] / np.sum(cm[0]) if cm.shape[0] > 1 else None
        cm_dict_reg[model_name] = cm

        reg_results[model_name] = {
            "Train Accuracy": train_acc,
            "Test Accuracy": test_acc,
            "F1-Macro": f1_score(y_test, y_test_pred, average="macro"),
            "Precision": precision_score(y_test, y_test_pred, average="macro"),
            "Recall": recall_score(y_test, y_test_pred, average="macro"),
            "MCC": matthews_corrcoef(y_test, y_test_pred),
            "AUC-PR": auc_pr,
            "AUC-ROC": auc_roc,
            "Specificity": specificity,
            "CV F1 Mean": np.nanmean(cv_scores),
            "CV F1 Std": np.nanstd(cv_scores),
            "Fit Status": fit_status
        }

        all_models[f"{model_name} - Regularised"] = reg_model

        print_summary(
            name=model_name + " (Regularised)",
            model=reg_model,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
            y_test=y_test,
            best_params=reg_model.get_params(),
            fit_status=fit_status,
            auc_pr=auc_pr,
            auc_roc=auc_roc,
            mcc=matthews_corrcoef(y_test, y_test_pred),
            specificity=specificity
        )

        # === CONDITIONAL TUNING ===
        if model_name in ["XGBoost", "Gradient Boosting", "LightGBM"]:
            print(f"Manually tuning {model_name}...")
            best_score = -np.inf
            best_model = None
            best_params = None

            for combo in product(*fine_tune_grids[model_name].values()):
                params = dict(zip(fine_tune_grids[model_name].keys(), combo))
                try:
                    tuned_model = clone(base_model).set_params(**params)
                    tuned_model.fit(X_train, y_train)
                    preds = tuned_model.predict(X_test)
                    score = f1_score(y_test, preds, average="macro")
                    if score > best_score:
                        best_score = score
                        best_model = tuned_model
                        best_params = params
                except Exception as e:
                    print(f"Skipping {params} due to error: {e}")

        else:
            print(f"Fine-tuning {model_name} using HalvingGridSearchCV...")
            halving_grid = HalvingGridSearchCV(
                estimator=clone(base_model),
                param_grid=fine_tune_grids[model_name],
                scoring="f1_macro",
                cv=cv,
                n_jobs=-1,
                factor=2,
                verbose=0
            )
            halving_grid.fit(X_train, y_train)
            best_model = halving_grid.best_estimator_
            best_params = halving_grid.best_params_

        y_train_ft = best_model.predict(X_train)
        y_test_ft = best_model.predict(X_test)
        train_acc_ft = accuracy_score(y_train, y_train_ft)
        test_acc_ft = accuracy_score(y_test, y_test_ft)
        fit_status_ft = (
            "Potential Overfitting Detected!" if train_acc_ft > test_acc_ft + 0.1 else
            "Potential Underfitting Detected!" if test_acc_ft > train_acc_ft + 0.1 else
            "Balanced Performance"
        )

        auc_roc_ft = auc_pr_ft = None
        if hasattr(best_model, "predict_proba"):
            try:
                y_probs_ft = best_model.predict_proba(X_test)
                auc_roc_ft = roc_auc_score(y_test, y_probs_ft, multi_class="ovr")
                auc_pr_ft = np.mean([
                    auc(*precision_recall_curve((y_test == i).astype(int), y_probs_ft[:, i])[1::-1])
                    for i in np.unique(y_test)
                ])
            except Exception:
                pass

        cm_ft = confusion_matrix(y_test, y_test_ft)
        specificity_ft = cm_ft[0, 0] / cm_ft[0].sum() if cm_ft.shape[0] > 1 else None
        cm_dict_ft[model_name] = cm_ft

        ft_results[model_name] = {
            "Best Params": best_params,
            "Train Accuracy": train_acc_ft,
            "Test Accuracy": test_acc_ft,
            "F1-Macro": f1_score(y_test, y_test_ft, average="macro"),
            "Precision": precision_score(y_test, y_test_ft, average="macro"),
            "Recall": recall_score(y_test, y_test_ft, average="macro"),
            "MCC": matthews_corrcoef(y_test, y_test_ft),
            "AUC-PR": auc_pr_ft,
            "AUC-ROC": auc_roc_ft,
            "CV F1 Mean": np.nan,
            "CV F1 Std": np.nan,
            "Fit Status": fit_status_ft
        }

        all_models[f"{model_name} - FineTuned"] = best_model

        print_summary(
            name=model_name + " (Fine-Tuned)",
            model=best_model,
            X_train=X_train,
            y_train=y_train,
            X_test=X_test,
            y_test=y_test,
            best_params=best_params,
            fit_status=fit_status_ft,
            auc_pr=auc_pr_ft,
            auc_roc=auc_roc_ft,
            mcc=matthews_corrcoef(y_test, y_test_ft),
            specificity=specificity_ft
        )

    df_reg = pd.DataFrame(reg_results).T
    df_ft = pd.DataFrame(ft_results).T

    print("\nSummary - Regularised Models:\n", df_reg)
    print("\nSummary - Fine-Tuned Models:\n", df_ft)

    return all_models, reg_results, ft_results, df_reg, df_ft, cm_dict_reg, cm_dict_ft
In [164]:
all_models, reg_results, ft_results, df_reg, df_ft, cm_dict_reg, cm_dict_ft = evaluate_pipeline_multiclass_with_manual_tuning(
    regularised_models=regularised_models,
    fine_tune_grids=fine_tune_grids,
    X_train=X_train_multi_dropped,
    y_train=y_train_multi,
    X_test=X_test_dropped,
    y_test=y_test_multi
)
Processing Decision Tree...

Starting grid search for Decision Tree (Regularised)...
              precision    recall  f1-score   support

           0       0.41      0.61      0.49      2683
           1       0.94      0.66      0.78     38015
           2       0.04      0.64      0.07       797
           3       0.64      0.70      0.67      5512
           4       0.50      0.19      0.27      5480

    accuracy                           0.61     52487
   macro avg       0.51      0.56      0.46     52487
weighted avg       0.83      0.61      0.69     52487


Results for Decision Tree (Regularised):
Best Parameters: {'ccp_alpha': 0.01, 'class_weight': None, 'criterion': 'gini', 'max_depth': 5, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'random_state': 42, 'splitter': 'best'}
Train Accuracy: 0.6192233691114365
Test Accuracy: 0.6116181149618001
F1-Score: 0.4564529763100015
AUC-PR: 0.5704715342960476
AUC-ROC: 0.8300965613486367
MCC: 0.4084862600275739
Specificity: 0.6067834513604174
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning Decision Tree using HalvingGridSearchCV...

Starting grid search for Decision Tree (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.41      0.61      0.49      2683
           1       0.94      0.66      0.78     38015
           2       0.04      0.64      0.07       797
           3       0.64      0.70      0.67      5512
           4       0.50      0.19      0.27      5480

    accuracy                           0.61     52487
   macro avg       0.51      0.56      0.46     52487
weighted avg       0.83      0.61      0.69     52487


Results for Decision Tree (Fine-Tuned):
Best Parameters: {'max_depth': 20, 'min_samples_split': 5}
Train Accuracy: 0.6192233691114365
Test Accuracy: 0.6116181149618001
F1-Score: 0.4564529763100015
AUC-PR: 0.5704715342960476
AUC-ROC: 0.8300965613486367
MCC: 0.4084862600275739
Specificity: 0.6067834513604174
Fit Status: Balanced Performance
--------------------------------------------------------------------------------

Processing Random Forest...

Starting grid search for Random Forest (Regularised)...
              precision    recall  f1-score   support

           0       0.49      0.57      0.53      2683
           1       0.96      0.77      0.85     38015
           2       0.05      0.61      0.09       797
           3       0.72      0.85      0.78      5512
           4       0.52      0.22      0.31      5480

    accuracy                           0.71     52487
   macro avg       0.55      0.60      0.51     52487
weighted avg       0.85      0.71      0.76     52487


Results for Random Forest (Regularised):
Best Parameters: {'bootstrap': True, 'ccp_alpha': 0.0, 'class_weight': None, 'criterion': 'gini', 'max_depth': 8, 'max_features': 'sqrt', 'max_leaf_nodes': None, 'max_samples': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 5, 'min_samples_split': 10, 'min_weight_fraction_leaf': 0.0, 'monotonic_cst': None, 'n_estimators': 100, 'n_jobs': None, 'oob_score': False, 'random_state': 42, 'verbose': 0, 'warm_start': False}
Train Accuracy: 0.6853664734568424
Test Accuracy: 0.7064987520719416
F1-Score: 0.5117880154629761
AUC-PR: 0.5763380024301908
AUC-ROC: 0.909100673499273
MCC: 0.5060311414345766
Specificity: 0.5736116287737607
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning Random Forest using HalvingGridSearchCV...

Starting grid search for Random Forest (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.45      0.52      0.48      2683
           1       0.96      0.84      0.89     38015
           2       0.05      0.14      0.07       797
           3       0.73      0.85      0.79      5512
           4       0.40      0.54      0.46      5480

    accuracy                           0.78     52487
   macro avg       0.52      0.58      0.54     52487
weighted avg       0.83      0.78      0.80     52487


Results for Random Forest (Fine-Tuned):
Best Parameters: {'max_depth': None, 'min_samples_split': 5, 'n_estimators': 200}
Train Accuracy: 0.9818033858105737
Test Accuracy: 0.7814315925848305
F1-Score: 0.538167899107699
AUC-PR: 0.5670788669572466
AUC-ROC: 0.9069611125710955
MCC: 0.5786120599903005
Specificity: 0.5184494968319046
Fit Status: Potential Overfitting Detected!
--------------------------------------------------------------------------------

Processing XGBoost...
Cross-validation failed: 'super' object has no attribute '__sklearn_tags__'

Starting grid search for XGBoost (Regularised)...
              precision    recall  f1-score   support

           0       0.50      0.55      0.52      2683
           1       0.96      0.79      0.87     38015
           2       0.05      0.59      0.09       797
           3       0.71      0.85      0.78      5512
           4       0.51      0.25      0.33      5480

    accuracy                           0.72     52487
   macro avg       0.55      0.60      0.52     52487
weighted avg       0.85      0.72      0.77     52487


Results for XGBoost (Regularised):
Best Parameters: {'objective': 'multi:softprob', 'base_score': None, 'booster': None, 'callbacks': None, 'colsample_bylevel': None, 'colsample_bynode': None, 'colsample_bytree': 0.8, 'device': None, 'early_stopping_rounds': None, 'enable_categorical': False, 'eval_metric': 'mlogloss', 'feature_types': None, 'gamma': None, 'grow_policy': None, 'importance_type': None, 'interaction_constraints': None, 'learning_rate': 0.05, 'max_bin': None, 'max_cat_threshold': None, 'max_cat_to_onehot': None, 'max_delta_step': None, 'max_depth': 4, 'max_leaves': None, 'min_child_weight': None, 'missing': nan, 'monotone_constraints': None, 'multi_strategy': None, 'n_estimators': 50, 'n_jobs': None, 'num_parallel_tree': None, 'random_state': 42, 'reg_alpha': None, 'reg_lambda': None, 'sampling_method': None, 'scale_pos_weight': None, 'subsample': 0.8, 'tree_method': None, 'validate_parameters': None, 'verbosity': 0, 'num_class': 5, 'use_label_encoder': False}
Train Accuracy: 0.6779879959825265
Test Accuracy: 0.7227885000095262
F1-Score: 0.5180339947021662
AUC-PR: 0.5787849701545823
AUC-ROC: 0.9093553276540376
MCC: 0.5188568430201828
Specificity: 0.5501304509877003
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Manually tuning XGBoost...

Starting grid search for XGBoost (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.49      0.54      0.51      2683
           1       0.96      0.81      0.88     38015
           2       0.05      0.42      0.09       797
           3       0.72      0.86      0.78      5512
           4       0.47      0.37      0.41      5480

    accuracy                           0.75     52487
   macro avg       0.54      0.60      0.53     52487
weighted avg       0.85      0.75      0.79     52487


Results for XGBoost (Fine-Tuned):
Best Parameters: {'n_estimators': 100, 'max_depth': 7, 'learning_rate': 0.1}
Train Accuracy: 0.7494524377110081
Test Accuracy: 0.7455560424486063
F1-Score: 0.5346915593883044
AUC-PR: 0.5812320297583501
AUC-ROC: 0.9129681551568705
MCC: 0.5426292518560163
Specificity: 0.5363399180022363
Fit Status: Balanced Performance
--------------------------------------------------------------------------------

Processing LightGBM...
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010752 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334

Starting grid search for LightGBM (Regularised)...
              precision    recall  f1-score   support

           0       0.51      0.54      0.52      2683
           1       0.96      0.79      0.87     38015
           2       0.05      0.55      0.09       797
           3       0.72      0.85      0.78      5512
           4       0.50      0.29      0.37      5480

    accuracy                           0.73     52487
   macro avg       0.55      0.61      0.53     52487
weighted avg       0.85      0.73      0.78     52487


Results for LightGBM (Regularised):
Best Parameters: {'boosting_type': 'gbdt', 'class_weight': None, 'colsample_bytree': 0.8, 'importance_type': 'split', 'learning_rate': 0.05, 'max_depth': 6, 'min_child_samples': 20, 'min_child_weight': 0.001, 'min_split_gain': 0.0, 'n_estimators': 50, 'n_jobs': None, 'num_leaves': 31, 'objective': 'multiclass', 'random_state': 42, 'reg_alpha': 0.0, 'reg_lambda': 0.0, 'subsample': 0.8, 'subsample_for_bin': 200000, 'subsample_freq': 0, 'num_class': 5}
Train Accuracy: 0.700788973729111
Test Accuracy: 0.7261036066073504
F1-Score: 0.5264928694119189
AUC-PR: 0.5816104379567691
AUC-ROC: 0.9126329537478147
MCC: 0.5254275158890521
Specificity: 0.5400670890793887
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Manually tuning LightGBM...
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009195 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.011043 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009226 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009780 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.011821 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010522 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Warning] No further splits with positive gain, best gain: -inf
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.009664 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334
[LightGBM] [Info] Auto-choosing col-wise multi-threading, the overhead of testing was 0.010194 seconds.
You can set `force_col_wise=true` to remove the overhead.
[LightGBM] [Info] Total Bins 1530
[LightGBM] [Info] Number of data points in the train set: 330556, number of used features: 6
[LightGBM] [Info] Start training from score -1.600811
[LightGBM] [Info] Start training from score -1.657514
[LightGBM] [Info] Start training from score -1.586843
[LightGBM] [Info] Start training from score -1.593252
[LightGBM] [Info] Start training from score -1.610334

Starting grid search for LightGBM (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.50      0.54      0.52      2683
           1       0.96      0.79      0.87     38015
           2       0.05      0.50      0.09       797
           3       0.73      0.85      0.79      5512
           4       0.50      0.34      0.40      5480

    accuracy                           0.73     52487
   macro avg       0.55      0.60      0.53     52487
weighted avg       0.85      0.73      0.78     52487


Results for LightGBM (Fine-Tuned):
Best Parameters: {'n_estimators': 50, 'max_depth': 5, 'learning_rate': 0.1}
Train Accuracy: 0.7058955214850131
Test Accuracy: 0.734581896469602
F1-Score: 0.5338048635612594
AUC-PR: 0.5828284215169861
AUC-ROC: 0.9138955036568385
MCC: 0.5340369506312618
Specificity: 0.5408125232948192
Fit Status: Balanced Performance
--------------------------------------------------------------------------------

Processing Gradient Boosting...

Starting grid search for Gradient Boosting (Regularised)...
              precision    recall  f1-score   support

           0       0.50      0.54      0.52      2683
           1       0.96      0.78      0.86     38015
           2       0.05      0.48      0.08       797
           3       0.73      0.85      0.78      5512
           4       0.48      0.35      0.41      5480

    accuracy                           0.72     52487
   macro avg       0.54      0.60      0.53     52487
weighted avg       0.85      0.72      0.78     52487


Results for Gradient Boosting (Regularised):
Best Parameters: {'ccp_alpha': 0.0, 'criterion': 'friedman_mse', 'init': None, 'learning_rate': 0.05, 'loss': 'log_loss', 'max_depth': 4, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'n_estimators': 100, 'n_iter_no_change': None, 'random_state': 42, 'subsample': 0.8, 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': 0, 'warm_start': False}
Train Accuracy: 0.7070723266254432
Test Accuracy: 0.7235124888067521
F1-Score: 0.5314326045798792
AUC-PR: 0.5801242007980838
AUC-ROC: 0.9107400069225646
MCC: 0.5250754674453689
Specificity: 0.5445396943719717
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Manually tuning Gradient Boosting...

Starting grid search for Gradient Boosting (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.49      0.55      0.52      2683
           1       0.96      0.79      0.87     38015
           2       0.05      0.40      0.08       797
           3       0.72      0.86      0.78      5512
           4       0.45      0.39      0.42      5480

    accuracy                           0.74     52487
   macro avg       0.53      0.60      0.54     52487
weighted avg       0.85      0.74      0.78     52487


Results for Gradient Boosting (Fine-Tuned):
Best Parameters: {'n_estimators': 200, 'max_depth': 3, 'learning_rate': 0.1}
Train Accuracy: 0.7233691114364889
Test Accuracy: 0.7393259283251091
F1-Score: 0.5351175033518946
AUC-PR: 0.5803869487166038
AUC-ROC: 0.911762257502688
MCC: 0.538531361066165
Specificity: 0.5516213194185613
Fit Status: Balanced Performance
--------------------------------------------------------------------------------

Processing Logistic Regression...

Starting grid search for Logistic Regression (Regularised)...
              precision    recall  f1-score   support

           0       0.44      0.56      0.50      2683
           1       0.96      0.76      0.85     38015
           2       0.05      0.52      0.09       797
           3       0.67      0.83      0.74      5512
           4       0.43      0.28      0.34      5480

    accuracy                           0.70     52487
   macro avg       0.51      0.59      0.50     52487
weighted avg       0.83      0.70      0.75     52487


Results for Logistic Regression (Regularised):
Best Parameters: {'C': 0.05, 'class_weight': None, 'dual': False, 'fit_intercept': True, 'intercept_scaling': 1, 'l1_ratio': None, 'max_iter': 1000, 'multi_class': 'multinomial', 'n_jobs': None, 'penalty': 'l2', 'random_state': 42, 'solver': 'lbfgs', 'tol': 0.0001, 'verbose': 0, 'warm_start': False}
Train Accuracy: 0.657849199530488
Test Accuracy: 0.7021738716253548
F1-Score: 0.5022068257562184
AUC-PR: 0.5547073323080622
AUC-ROC: 0.8983600298483101
MCC: 0.49541005283530465
Specificity: 0.5628028326500186
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning Logistic Regression using HalvingGridSearchCV...

Starting grid search for Logistic Regression (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.44      0.56      0.50      2683
           1       0.96      0.76      0.85     38015
           2       0.05      0.52      0.09       797
           3       0.67      0.83      0.74      5512
           4       0.43      0.28      0.34      5480

    accuracy                           0.70     52487
   macro avg       0.51      0.59      0.50     52487
weighted avg       0.83      0.70      0.75     52487


Results for Logistic Regression (Fine-Tuned):
Best Parameters: {'C': 0.1, 'penalty': 'l2'}
Train Accuracy: 0.6578522247365045
Test Accuracy: 0.7020786099415093
F1-Score: 0.5021753711489946
AUC-PR: 0.5546720448568512
AUC-ROC: 0.8983543789694515
MCC: 0.49532214540077857
Specificity: 0.5628028326500186
Fit Status: Balanced Performance
--------------------------------------------------------------------------------

Processing MLP...

Starting grid search for MLP (Regularised)...
              precision    recall  f1-score   support

           0       0.41      0.53      0.46      2683
           1       0.96      0.79      0.87     38015
           2       0.05      0.29      0.08       797
           3       0.70      0.85      0.77      5512
           4       0.37      0.43      0.40      5480

    accuracy                           0.74     52487
   macro avg       0.50      0.58      0.52     52487
weighted avg       0.83      0.74      0.78     52487


Results for MLP (Regularised):
Best Parameters: {'activation': 'tanh', 'alpha': 0.01, 'batch_size': 'auto', 'beta_1': 0.9, 'beta_2': 0.999, 'early_stopping': True, 'epsilon': 1e-08, 'hidden_layer_sizes': (50, 50), 'learning_rate': 'constant', 'learning_rate_init': 0.001, 'max_fun': 15000, 'max_iter': 300, 'momentum': 0.9, 'n_iter_no_change': 10, 'nesterovs_momentum': True, 'power_t': 0.5, 'random_state': 42, 'shuffle': True, 'solver': 'adam', 'tol': 0.0001, 'validation_fraction': 0.1, 'verbose': False, 'warm_start': False}
Train Accuracy: 0.7733424896235433
Test Accuracy: 0.7369253338922018
F1-Score: 0.5162021263461105
AUC-PR: 0.5452388092679551
AUC-ROC: 0.8938913842139298
MCC: 0.5296972751558054
Specificity: 0.5285128587402161
Fit Status: Balanced Performance
--------------------------------------------------------------------------------
Fine-tuning MLP using HalvingGridSearchCV...

Starting grid search for MLP (Fine-Tuned)...
              precision    recall  f1-score   support

           0       0.41      0.51      0.46      2683
           1       0.96      0.77      0.86     38015
           2       0.04      0.34      0.07       797
           3       0.76      0.82      0.79      5512
           4       0.38      0.45      0.41      5480

    accuracy                           0.72     52487
   macro avg       0.51      0.58      0.52     52487
weighted avg       0.84      0.72      0.77     52487


Results for MLP (Fine-Tuned):
Best Parameters: {'activation': 'tanh', 'hidden_layer_sizes': (50, 50), 'learning_rate_init': 0.01}
Train Accuracy: 0.7600104067086969
Test Accuracy: 0.7235124888067521
F1-Score: 0.5179979637492016
AUC-PR: 0.5462513604457632
AUC-ROC: 0.8901735492278563
MCC: 0.5178724866426806
Specificity: 0.5124860231084607
Fit Status: Balanced Performance
--------------------------------------------------------------------------------

Summary - Regularised Models:
                     Train Accuracy Test Accuracy  F1-Macro Precision  \
Decision Tree             0.619223      0.611618  0.456453  0.507176   
Random Forest             0.685366      0.706499  0.511788  0.547421   
XGBoost                   0.677988      0.722789  0.518034  0.546501   
LightGBM                  0.700789      0.726104  0.526493   0.54923   
Gradient Boosting         0.707072      0.723512  0.531433  0.543546   
Logistic Regression       0.657849      0.702174  0.502207  0.510532   
MLP                       0.773342      0.736925  0.516202   0.49961   

                       Recall       MCC    AUC-PR   AUC-ROC Specificity  \
Decision Tree         0.56017  0.408486  0.570472  0.830097    0.606783   
Random Forest        0.603762  0.506031  0.576338  0.909101    0.573612   
XGBoost              0.604678  0.518857  0.578785  0.909355     0.55013   
LightGBM             0.605305  0.525428   0.58161  0.912633    0.540067   
Gradient Boosting    0.601762  0.525075  0.580124   0.91074     0.54454   
Logistic Regression  0.590926   0.49541  0.554707   0.89836    0.562803   
MLP                  0.578315  0.529697  0.545239  0.893891    0.528513   

                    CV F1 Mean CV F1 Std            Fit Status  
Decision Tree         0.613008  0.001131  Balanced Performance  
Random Forest         0.673387  0.001453  Balanced Performance  
XGBoost                    NaN       NaN  Balanced Performance  
LightGBM              0.692176   0.00124  Balanced Performance  
Gradient Boosting     0.701366  0.001139  Balanced Performance  
Logistic Regression   0.651506  0.001723  Balanced Performance  
MLP                   0.769928  0.003396  Balanced Performance  

Summary - Fine-Tuned Models:
                                                            Best Params  \
Decision Tree                {'max_depth': 20, 'min_samples_split': 5}   
Random Forest        {'max_depth': None, 'min_samples_split': 5, 'n...   
XGBoost              {'n_estimators': 100, 'max_depth': 7, 'learnin...   
LightGBM             {'n_estimators': 50, 'max_depth': 5, 'learning...   
Gradient Boosting    {'n_estimators': 200, 'max_depth': 3, 'learnin...   
Logistic Regression                        {'C': 0.1, 'penalty': 'l2'}   
MLP                  {'activation': 'tanh', 'hidden_layer_sizes': (...   

                    Train Accuracy Test Accuracy  F1-Macro Precision  \
Decision Tree             0.619223      0.611618  0.456453  0.507176   
Random Forest             0.981803      0.781432  0.538168  0.517211   
XGBoost                   0.749452      0.745556  0.534692  0.538352   
LightGBM                  0.705896      0.734582  0.533805   0.54871   
Gradient Boosting         0.723369      0.739326  0.535118   0.53493   
Logistic Regression       0.657852      0.702079  0.502175  0.510491   
MLP                        0.76001      0.723512  0.517998  0.512388   

                       Recall       MCC    AUC-PR   AUC-ROC CV F1 Mean  \
Decision Tree         0.56017  0.408486  0.570472  0.830097        NaN   
Random Forest        0.575766  0.578612  0.567079  0.906961        NaN   
XGBoost              0.597378  0.542629  0.581232  0.912968        NaN   
LightGBM             0.604137  0.534037  0.582828  0.913896        NaN   
Gradient Boosting    0.599081  0.538531  0.580387  0.911762        NaN   
Logistic Regression  0.590899  0.495322  0.554672  0.898354        NaN   
MLP                  0.576717  0.517872  0.546251  0.890174        NaN   

                    CV F1 Std                       Fit Status  
Decision Tree             NaN             Balanced Performance  
Random Forest             NaN  Potential Overfitting Detected!  
XGBoost                   NaN             Balanced Performance  
LightGBM                  NaN             Balanced Performance  
Gradient Boosting         NaN             Balanced Performance  
Logistic Regression       NaN             Balanced Performance  
MLP                       NaN             Balanced Performance  
In [187]:
import os
import joblib

# Save path
export_path = r"C:\Users\ghaza\Desktop\FYP\Application\AI\models\Finetuned_Reg_Multiclass"
os.makedirs(export_path, exist_ok=True)

# Save models with clean filenames
for model_key, model in all_models.items():
    # Clean filename: lowercase, replace spaces with underscores
    filename = model_key.lower().replace(" ", "_").replace("-", "").replace("__", "_") + ".pkl"
    filepath = os.path.join(export_path, filename)
    joblib.dump(model, filepath)

print(f"Exported {len(all_models)} models to: {export_path}")
Exported 14 models to: C:\Users\ghaza\Desktop\FYP\Application\AI\models\Finetuned_Reg_Multiclass
In [167]:
df_reg.index = [f"{name} (Reg)" for name in df_reg.index]
df_ft.index = [f"{name} (Tuned)" for name in df_ft.index]
combined_df = pd.concat([df_reg, df_ft])
In [182]:
plot_model_metric_bars(combined_df)
No description has been provided for this image
Skipping F1-Macro: missing columns.
No description has been provided for this image
No description has been provided for this image
In [169]:
plot_roc_pr_curves(
    model_dict=all_models,
    X_test=X_test_dropped,
    y_test=y_test_multi,
    title_prefix="Regularlised vs Halving Grid Search"
)
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [170]:
shap_values_main = explain_models_with_shap(
    models=all_models,
    X_train=X_train_multi_dropped,
    X_test=X_test_dropped,
    sample_size=100,
    background_size=100
)

plot_mean_shap_summary(
    shap_values_main=shap_values_main,
    feature_names=X_train_multi_dropped.columns
)
SHAP for: Decision Tree - Regularised
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Decision Tree - FineTuned
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Random Forest - Regularised
No description has been provided for this image
SHAP for: Random Forest - FineTuned
No description has been provided for this image
SHAP for: XGBoost - Regularised
No description has been provided for this image
SHAP for: XGBoost - FineTuned
No description has been provided for this image
SHAP for: LightGBM - Regularised
No description has been provided for this image
SHAP for: LightGBM - FineTuned
No description has been provided for this image
SHAP for: Gradient Boosting - Regularised
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Gradient Boosting - FineTuned
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Logistic Regression - Regularised
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: Logistic Regression - FineTuned
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: MLP - Regularised
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
SHAP for: MLP - FineTuned
  0%|          | 0/100 [00:00<?, ?it/s]
No description has been provided for this image
No description has been provided for this image
In [165]:
learning_curve_data_dict = {}

for model_name, model in all_models.items():
    print(f"Generating learning curve for: {model_name}")
    try:
        df_lc = get_learning_curve_data(
            estimator=model,
            X=X_train_multi_dropped,
            y=y_train_multi,
            cv=cv_strategy,
            scoring="f1_macro"
        )
        learning_curve_data_dict[model_name] = df_lc
    except Exception as e:
        print(f"Failed for {model_name}: {e}")
Generating learning curve for: Decision Tree - Regularised
Generating learning curve for: Decision Tree - FineTuned
Generating learning curve for: Random Forest - Regularised
Generating learning curve for: Random Forest - FineTuned
Generating learning curve for: XGBoost - Regularised
Failed for XGBoost - Regularised: 'super' object has no attribute '__sklearn_tags__'
Generating learning curve for: XGBoost - FineTuned
Failed for XGBoost - FineTuned: 'super' object has no attribute '__sklearn_tags__'
Generating learning curve for: LightGBM - Regularised
Generating learning curve for: LightGBM - FineTuned
Generating learning curve for: Gradient Boosting - Regularised
Generating learning curve for: Gradient Boosting - FineTuned
Generating learning curve for: Logistic Regression - Regularised
Generating learning curve for: Logistic Regression - FineTuned
Generating learning curve for: MLP - Regularised
Generating learning curve for: MLP - FineTuned
In [180]:
plot_combined_learning_curves(
    learning_curve_data_dict=learning_curve_data_dict,
    title_prefix=" "
)
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image
In [202]:
# Step 1: Extract fitted models (these MUST be fitted scikit-learn classifiers)
xgb_model = all_models.get("XGBoost - FineTuned")
lgbm_model = all_models.get("LightGBM - FineTuned")
gb_model = all_models.get("Gradient Boosting - FineTuned")

# Step 2: Verify they are valid estimators
print(f"xgb -> {type(xgb_model)}")
print(f"lgbm -> {type(lgbm_model)}")
print(f"gb   -> {type(gb_model)}")
xgb -> <class 'xgboost.sklearn.XGBClassifier'>
lgbm -> <class 'lightgbm.sklearn.LGBMClassifier'>
gb   -> <class 'sklearn.ensemble._gb.GradientBoostingClassifier'>
In [204]:
import sklearn
import xgboost
import lightgbm

print("scikit-learn:", sklearn.__version__)
print("xgboost:", xgboost.__version__)
print("lightgbm:", lightgbm.__version__)
scikit-learn: 1.6.1
xgboost: 2.1.1
lightgbm: 4.6.0
In [ ]: